8import scrapy
import re
import json
from ..items import FiscalDataItem

class FiscalDataSpider(scrapy.Spider):
    name = 'fiscal_data'
    allowed_domains = ['mof.gov.cn', 'gks.mof.gov.cn']
    start_urls = [
        # 全国财政收支数据
        'https://www.mof.gov.cn/zhengwuxinxi/redianzhuanti/quanguocaizhengshouzhiqingkuang/',
        # 财政数据统计专页
        'https://www.mof.gov.cn/gkml/caizhengshuju/',
        # 国库司统计数据页
        'https://gks.mof.gov.cn/tongjishuju/'
    ]
    
    def parse(self, response):
        self.logger.info('正在爬取财政数据页面: %s', response.url)
        
        # 获取财政收支数据链接
        links = response.css('a::attr(href)').getall()
        # 筛选含有财政收支关键词的链接
        relevant_links = []
        for link in links:
            # 筛选财政收支、税收数据、政府基金相关链接
            if any(keyword in link for keyword in ['shouz', 'caizhengshouz', 'jijin']):
                relevant_links.append(link)
            elif link.endswith('.htm') and any(keyword in link for keyword in ['月财政', '季度财政', '年财政']):
                relevant_links.append(link)
        
        # 处理并跟踪链接
        for link in relevant_links:
            if link.startswith('./'):
                link = link[2:]
            if not link.startswith('http'):
                if link.startswith('/'):
                    if 'gks.mof.gov.cn' in response.url:
                        link = f'https://gks.mof.gov.cn{link}'
                    else:
                        link = f'https://www.mof.gov.cn{link}'
                else:
                    base_url = '/'.join(response.url.split('/')[:-1])
                    link = f'{base_url}/{link}'
            
            yield response.follow(link, callback=self.parse_data_page)
        
        # 查找分页链接
        next_page = response.css('a:contains("下一页")::attr(href), a.next::attr(href)').get()
        if next_page:
            if not next_page.startswith('http'):
                base_url = '/'.join(response.url.split('/')[:-1])
                next_page = f'{base_url}/{next_page}'
            yield response.follow(next_page, callback=self.parse)
    
    def parse_data_page(self, response):
        self.logger.info('正在解析财政数据详情页: %s', response.url)
        
        item = FiscalDataItem()
        
        # 提取标题
        item['title'] = response.css('div.xilan_tit::text, div.the_title::text, h2::text').get() or \
                        response.xpath('//title/text()').get()
        
        if item['title']:
            item['title'] = item['title'].strip()
        
        # 提取发布日期
        date_pattern = r'\d{4}[-年]\d{1,2}[-月]\d{1,2}'
        date_text = response.css('div.xilan_titf::text, div.the_title_f::text, div.pages-date::text').get() or ''
        date_match = re.search(date_pattern, date_text)
        if date_match:
            item['publish_date'] = date_match.group(0).replace('年', '-').replace('月', '-').replace('日', '')
        else:
            item['publish_date'] = None
        
        # 提取内容
        content_selectors = [
            'div#zoom p, div#zoom table',
            'div.TRS_Editor p, div.TRS_Editor table',
            'div.xilan_con p, div.xilan_con table',
            'div.the_content p, div.the_content table',
        ]
        
        content_html = ''
        for selector in content_selectors:
            content_elements = response.css(selector)
            if content_elements:
                content_html = ''.join(element.get() for element in content_elements)
                break
        
        item['content_html'] = content_html
        
        # 提取纯文本内容（用于保存到数据库）
        item['content'] = ' '.join(response.css('div#zoom p::text, div.TRS_Editor p::text, div.xilan_con p::text, div.the_content p::text').getall()).strip()
        
        # 提取数据表格
        tables = response.css('table')
        item['tables'] = []
        
        for table in tables:
            table_data = []
            rows = table.css('tr')
            for row in rows:
                cols = row.css('td, th')
                row_data = [col.css('::text').get() or '' for col in cols]
                row_data = [text.strip() for text in row_data]
                table_data.append(row_data)
            
            if table_data:
                item['tables'].append(table_data)
        
        # 提取财政收入数据
        item['revenue_data'] = self.extract_revenue_data(item['content'])
        
        # 提取财政支出数据
        item['expenditure_data'] = self.extract_expenditure_data(item['content'])
        
        # 提取政府性基金数据
        item['government_fund_data'] = self.extract_government_fund_data(item['content'])
        
        # 添加URL
        item['url'] = response.url
        
        # 分类
        if '月' in item['title']:
            item['period_type'] = '月度'
        elif '季度' in item['title']:
            item['period_type'] = '季度'
        elif '年' in item['title']:
            item['period_type'] = '年度'
        else:
            item['period_type'] = '其他'
        
        return item
    
    def extract_revenue_data(self, content):
        """从内容中提取财政收入数据"""
        revenue_data = {}
        
        # 全国一般公共预算收入
        total_revenue_pattern = r'全国一般公共预算收入(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'
        total_revenue_match = re.search(total_revenue_pattern, content)
        if total_revenue_match:
            amount = int(total_revenue_match.group(1))
            direction = total_revenue_match.group(2)
            change_rate = float(total_revenue_match.group(3))
            revenue_data['total_revenue'] = {
                'amount': amount,
                'change_rate': change_rate if direction == '增长' else -change_rate
            }
        
        # 税收收入
        tax_revenue_pattern = r'税收收入(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'
        tax_revenue_match = re.search(tax_revenue_pattern, content)
        if tax_revenue_match:
            amount = int(tax_revenue_match.group(1))
            direction = tax_revenue_match.group(2)
            change_rate = float(tax_revenue_match.group(3))
            revenue_data['tax_revenue'] = {
                'amount': amount,
                'change_rate': change_rate if direction == '增长' else -change_rate
            }
        
        # 非税收入
        non_tax_revenue_pattern = r'非税收入(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'
        non_tax_revenue_match = re.search(non_tax_revenue_pattern, content)
        if non_tax_revenue_match:
            amount = int(non_tax_revenue_match.group(1))
            direction = non_tax_revenue_match.group(2)
            change_rate = float(non_tax_revenue_match.group(3))
            revenue_data['non_tax_revenue'] = {
                'amount': amount,
                'change_rate': change_rate if direction == '增长' else -change_rate
            }
        
        # 各税种收入
        tax_types = [
            ('domestic_vat', r'国内增值税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('domestic_consumption_tax', r'国内消费税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('corporate_income_tax', r'企业所得税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('personal_income_tax', r'个人所得税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('import_vat_consumption', r'进口货物增值税、消费税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('tariff', r'关税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('export_rebate', r'出口退税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('urban_maintenance_tax', r'城市维护建设税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('vehicle_purchase_tax', r'车辆购置税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('stamp_duty', r'印花税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('stock_trading_stamp_duty', r'证券交易印花税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('resource_tax', r'资源税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('deed_tax', r'契税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('property_tax', r'房产税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('land_use_tax', r'城镇土地使用税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('land_appreciation_tax', r'土地增值税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('farmland_occupation_tax', r'耕地占用税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('environmental_protection_tax', r'环境保护税(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%')
        ]
        
        for tax_key, tax_pattern in tax_types:
            tax_match = re.search(tax_pattern, content)
            if tax_match:
                amount = int(tax_match.group(1))
                direction = tax_match.group(2)
                change_rate = float(tax_match.group(3))
                revenue_data[tax_key] = {
                    'amount': amount,
                    'change_rate': change_rate if direction == '增长' else -change_rate
                }
        
        return revenue_data
    
    def extract_expenditure_data(self, content):
        """从内容中提取财政支出数据"""
        expenditure_data = {}
        
        # 全国一般公共预算支出
        total_expenditure_pattern = r'全国一般公共预算支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'
        total_expenditure_match = re.search(total_expenditure_pattern, content)
        if total_expenditure_match:
            amount = int(total_expenditure_match.group(1))
            direction = total_expenditure_match.group(2)
            change_rate = float(total_expenditure_match.group(3))
            expenditure_data['total_expenditure'] = {
                'amount': amount,
                'change_rate': change_rate if direction == '增长' else -change_rate
            }
        
        # 各类支出
        expense_types = [
            ('education', r'教育支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('science_technology', r'科学技术支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('culture_sports_media', r'文化旅游体育与传媒支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('social_security_employment', r'社会保障和就业支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('health', r'卫生健康支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('energy_environment', r'节能环保支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('urban_rural_community', r'城乡社区支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('agriculture', r'农林水支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('transportation', r'交通运输支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'),
            ('debt_interest', r'债务付息支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%')
        ]
        
        for expense_key, expense_pattern in expense_types:
            expense_match = re.search(expense_pattern, content)
            if expense_match:
                amount = int(expense_match.group(1))
                direction = expense_match.group(2)
                change_rate = float(expense_match.group(3))
                expenditure_data[expense_key] = {
                    'amount': amount,
                    'change_rate': change_rate if direction == '增长' else -change_rate
                }
        
        return expenditure_data
    
    def extract_government_fund_data(self, content):
        """从内容中提取政府性基金数据"""
        fund_data = {}
        
        # 全国政府性基金预算收入
        fund_revenue_pattern = r'全国政府性基金预算收入(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'
        fund_revenue_match = re.search(fund_revenue_pattern, content)
        if fund_revenue_match:
            amount = int(fund_revenue_match.group(1))
            direction = fund_revenue_match.group(2)
            change_rate = float(fund_revenue_match.group(3))
            fund_data['total_fund_revenue'] = {
                'amount': amount,
                'change_rate': change_rate if direction == '增长' else -change_rate
            }
        
        # 国有土地使用权出让收入
        land_revenue_pattern = r'国有土地使用权出让收入(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'
        land_revenue_match = re.search(land_revenue_pattern, content)
        if land_revenue_match:
            amount = int(land_revenue_match.group(1))
            direction = land_revenue_match.group(2)
            change_rate = float(land_revenue_match.group(3))
            fund_data['land_revenue'] = {
                'amount': amount,
                'change_rate': change_rate if direction == '增长' else -change_rate
            }
        
        # 全国政府性基金预算支出
        fund_expenditure_pattern = r'全国政府性基金预算支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'
        fund_expenditure_match = re.search(fund_expenditure_pattern, content)
        if fund_expenditure_match:
            amount = int(fund_expenditure_match.group(1))
            direction = fund_expenditure_match.group(2)
            change_rate = float(fund_expenditure_match.group(3))
            fund_data['total_fund_expenditure'] = {
                'amount': amount,
                'change_rate': change_rate if direction == '增长' else -change_rate
            }
        
        # 国有土地使用权出让收入相关支出
        land_expenditure_pattern = r'国有土地使用权出让收入相关支出(\d+)亿元.*?同比(增长|下降)(\d+\.?\d*)%'
        land_expenditure_match = re.search(land_expenditure_pattern, content)
        if land_expenditure_match:
            amount = int(land_expenditure_match.group(1))
            direction = land_expenditure_match.group(2)
            change_rate = float(land_expenditure_match.group(3))
            fund_data['land_expenditure'] = {
                'amount': amount,
                'change_rate': change_rate if direction == '增长' else -change_rate
            }
        
        return fund_data 